
from bs4 import BeautifulSoup #导入BeautifulSoup库
import requests #导入requests库，用来读取在线网页数据的
import time
### ex04：提取小猪租房（http://www.xiaozhu.com/） 的详细信息（多页面），这里以杭州短租房为例
##  网址：http://hz.xiaozhu.com/search-duanzufang-p1-0/，http://hz.xiaozhu.com/search-duanzufang-p2-0/（爬取5个页面）
##  时间：2017年10月12日
##  难点:批量获取链接

###第一个函数用于获取详情页网址
def get_detail_urls(number):
    urls = []
    for i in range(1,number):
        time.sleep(1) #不要频繁爬取
        url = "http://hz.xiaozhu.com/search-duanzufang-p{}-0/".format(i)  ##生成类似http://hz.xiaozhu.com/search-duanzufang-p1-0/的页面
        webdata = requests.get(url)#通过get方法获取网页数据
        soup = BeautifulSoup(webdata.text, 'lxml')  # 通过BeautifulSoup对网页数据解析
        links = soup.select("a.resule_img_a") #获取http://hz.xiaozhu.com/search-duanzufang-p1-0/的页面所有房子的详细网址链接
        for link in links:
            urls.append(link.get("href"))
    return urls

###第二个函数用于提取详情页的信息
def get_url_info(url):
    webdata = requests.get(url)  # 通过get方法获取网页数据
    soup = BeautifulSoup(webdata.text, 'lxml')  # 通过BeautifulSoup对网页数据解析
    name = soup.select("div.pho_info > h4 > em")[0].text  # 获取公寓名称
    address = soup.select("div.pho_info > p")[0].get("title")  # 获取地址
    price = soup.select("#pricePart > div.day_l > span")[0].text  # 获取价格
    image = soup.select("#curBigImage")[0].get("src")  # 获取图片
    lorder_name = soup.select("a.lorder_name")[0].get("title") #获取房东名字
    lorder_gender =soup.select("#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > span")[0].get('class')[0]
    data = {
            "房名":name,
            "地址":address,
            "价格":price+" 元",
            "图片":image,
            "房东":lorder_name,
            "性别":judge_gender(lorder_gender)
    }
    print(data)

### 判断性别
def judge_gender(class_name):
    if class_name == "member_girl_ico":
        return "女"
    else:
        return "男"

### 主函数
def main():
    detail_urls = get_detail_urls(5)
    for url in detail_urls:
        time.sleep(1)
        get_url_info(url)

main()
